
# coding: utf-8

# In[1]:


# 此文件的作用：
#
#
#
# 要求： 


# In[2]:


from lxml import etree
from tqdm import tqdm
import requests
import time
import shutil 
import os
import urllib.request

from PyQt5 import  QtWidgets
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
import sys
import threading
from threading import Event
import copy
import datetime


# In[3]:


ZUO_PIN_INFO_CONFIG = "zuo_pin_shu_from_dou_yin_.txt"
XIN_ZENG_INFO_CONFIG = "xin_zeng_zuo_pin_shu.txt"
LOG_FILE_PATH = "log.txt"


# In[4]:


class Logger(object):
    def __init__(self, filename="Default.log"):
        self.terminal = sys.stdout
        self.filename = filename
        
    def write(self, message):
        self.log = open(self.filename, "a", encoding="utf-8")  # 防止编码错误
        self.terminal.write(message)
        self.log.write(message)
        self.log.close()
    def flush(self):
        pass
    def reset(self):
        #self.log.close()
        sys.stdout=self.terminal


# In[5]:


sys.stdout = Logger(LOG_FILE_PATH) 


# In[6]:


print("98888")


# In[7]:


# 从文件中加载原有作品数
def load_zuo_pin_shu_from_file(zuo_pin_shu_save_path):
    zuo_pin_shu_of_homepage_list = []
    if not os.path.exists(zuo_pin_shu_save_path):
        return zuo_pin_shu_of_homepage_list
    file_read = open(zuo_pin_shu_save_path, "r", encoding='UTF-8')
    lines = file_read.readlines()
    for line in lines:
        line = line.strip()
        if len(line) < 4:
            continue
        data_list = line.split("\t")
        assert len(data_list) == 3, "{}内数据格式不合法".format(zuo_pin_shu_save_path)
        zuo_pin_shu_of_homepage_list.append((data_list[0], data_list[1], int(data_list[2])))
    file_read.close()
    print("从文件{}中，共读取到{}个主页的作品量数据".format(zuo_pin_shu_save_path, len(zuo_pin_shu_of_homepage_list)))
    return zuo_pin_shu_of_homepage_list

# 将作品数保存到文件中
def save_zuo_pin_shu_to_file(html_info_list, zuo_pin_shu_save_path):
    file_write = open(zuo_pin_shu_save_path, "w", encoding='UTF-8')
    for html_info in html_info_list:
        file_write.write("{}\t{}\t{}\n".format(html_info[0], html_info[1], html_info[2]))
    file_write.close()
    print("将作品数保存到文件{}中".format(zuo_pin_shu_save_path))
    return True

# 从文件中加载新增数记录
def load_xin_zeng_shu_from_file(xin_zeng_save_path):
    xin_zeng_info_list = []
    if not os.path.exists(xin_zeng_save_path):
        return xin_zeng_info_list
    file_read = open(xin_zeng_save_path, "r", encoding='UTF-8')
    lines = file_read.readlines()
    for line in lines:
        line = line.strip()
        if len(line) < 4:
            continue
        data_list = line.split("\t")
        assert len(data_list) == 3, "{}内数据格式不合法".format(xin_zeng_save_path)
        datetime_ = datetime.datetime.strptime(data_list[2],"%Y-%m-%d %H:%M:%S")
        #localtime_ = time.localtime(time.mktime(datetime_.timetuple()))
        xin_zeng_info_list.append((data_list[0], int(data_list[1]), datetime_))
    file_read.close()
    print("从文件{}中，共读取到{}条新增记录".format(xin_zeng_save_path, len(xin_zeng_info_list)))
    return xin_zeng_info_list

# 将新增作品数保存到文件中
def save_xin_zeng_shu_to_file(xin_zeng_info_list, xin_zeng_save_path):
    file_write = open(xin_zeng_save_path, "w", encoding='UTF-8')
    for xin_zeng_info in xin_zeng_info_list:
        #time_str = time.strftime('%Y-%m-%d %H:%M:%S', xin_zeng_info[2])  
        time_str = str(xin_zeng_info[2].strftime("%Y-%m-%d %H:%M:%S") )
        file_write.write("{}\t{}\t{}\n".format(xin_zeng_info[0], xin_zeng_info[1], time_str))
    file_write.close()
    print("将新增数保存到文件{}中".format(xin_zeng_save_path))
    return True


# In[8]:


g_html_info_list = load_zuo_pin_shu_from_file(ZUO_PIN_INFO_CONFIG)
g_xin_zeng_info_list = load_xin_zeng_shu_from_file(XIN_ZENG_INFO_CONFIG)


# In[9]:


g_xin_zeng_info_list


# In[10]:


def get_work_info(html_path):
    try:
        response = requests.get(html_path)
        #time.sleep(0.5)
        html_serial = etree.HTML(response.text)
        page_names = html_serial.xpath("//span/span")
        page_name = page_names[0].xpath("string(.)")
        #print("主页名:{}".format(page_name))
        user_tab_count = html_serial.xpath("//span[@data-e2e='user-tab-count']")
        if len(user_tab_count) not in [1, 2]:
            print("不合法的html:{}".format(user_tab_count))
            return -1, ""
        #print(user_tab_count[0])
        count_str = user_tab_count[0].xpath("string(.)")
        count_int = int(count_str)
        return count_int, page_name
    except Exception as e:
        print("！！！！获得的页面的数据不对，可能有反爬虫")
        print(e)
        return -1, ""
    return -1, ""


# In[11]:



def do_one_loop(html_info_list, input_event, table):
    for i, html_info in enumerate(html_info_list):
        html_path, _, _ = html_info
        count_int, page_name = get_work_info(html_path)
        if -1 == count_int:
            print("【{}】【{}】！！！！爬虫失败".format(i, html_path))
        else:
            print("【{}】【{}】作品数是:{}".format(i, html_path, count_int))
            if count_int > html_info_list[i][2] and html_info_list[i][2] == 0:   
                html_info_list[i] = (html_path, page_name, count_int)
                save_zuo_pin_shu_to_file(html_info_list, ZUO_PIN_INFO_CONFIG)
            if count_int > html_info_list[i][2] and html_info_list[i][2] != 0:
                print("^_^发现了新作品")
                xin_zeng_count = count_int - html_info_list[i][2]
                datetime_ = datetime.datetime.now()
                # time.localtime(time.time())
                g_xin_zeng_info_list.append((html_path, xin_zeng_count, datetime_))
                save_xin_zeng_shu_to_file(g_xin_zeng_info_list, XIN_ZENG_INFO_CONFIG)
                
                html_info_list[i] = (html_path, page_name, count_int)
                save_zuo_pin_shu_to_file(html_info_list, ZUO_PIN_INFO_CONFIG)
           
                
            #table.tableView.setEnabled(True)
            table.model.setItem(i, 1, QStandardItem(page_name)) 
            #table.tableView.setEnabled(False)
        
        if True == input_event.wait(1.5):
            print("do_one_loop, 被要求退出")
            break
        #time.sleep(1.5)
        
#i_loop_count = 0 
#while True:        
#    i_loop_count += 1
#    print("【{}】".format(i_loop_count))
#    do_one_loop(g_html_info_list)
#    time.sleep(20)
###


# In[12]:


# 更新新增作品数 
def update_xin_zeng_shu(table):
    global g_html_info_list
    
    #localtime_now = time.localtime(time.time())
    datetime_now = datetime.datetime.now()
    for i, html_info in enumerate(g_html_info_list):
        html_path, _, _ = html_info
        i_count = 0
        for xin_zeng_info in g_xin_zeng_info_list:
            html_url, xin_zeng_count, datetime_ = xin_zeng_info
            if html_url != html_path:
                continue
            intal_seconds = (datetime_now -datetime_).seconds
            if intal_seconds > 30*60:
                continue
            i_count += xin_zeng_count
        
        item = QStandardItem(str(i_count))
        if i_count > 0:
            item.setBackground(QBrush(QColor(255, 0, 0, 250)))
        else:
            item.setBackground(QBrush(QColor(255, 255, 255, 255)))
        item.setTextAlignment(Qt.AlignHCenter | Qt.AlignVCenter)
        table.model.setItem(i, 2, item) 

          


# In[13]:


g_Event = Event()


# In[14]:


def monitor_thread(table):
    print("===>线程名称:{} 参数：{} 开始时间:{}".format(threading.current_thread().name, table, time.strftime("%Y-%m-%d %H:%M%S")))
    print("监控的主页共{}个:{}".format(len(g_html_info_list), g_html_info_list))
    
    i_loop_count = 0  
    while True:        
        i_loop_count += 1
        print("【{}】".format(i_loop_count))
        do_one_loop(g_html_info_list, g_Event, table)
        # 更新新增作品数 
        update_xin_zeng_shu(table)
   
        if True == g_Event.wait(20):
            print("monitor_thread, 被要求退出")
            break
        #time.sleep(20)
    
    print("<===线程名称:{} 开始时间:{}".format(threading.current_thread().name, time.strftime("%Y-%m-%d %H:%M%S")))
    
    return


# In[15]:


class ReadOnlyDelegate(QtWidgets.QStyledItemDelegate):
    def createEditor(self, parent, option, index):
        return


# In[16]:



class Table(QWidget):
 
	def __init__(self, arg=None):
		super(Table, self).__init__(arg)
		self.setWindowTitle("抖音新作品监控")
        
		operatorLayout = QHBoxLayout()
		self.startBtn = QPushButton("开始监控")
		self.stopBtn = QPushButton("停止监控")
		self.addBtn = QPushButton("添加主页")
		self.delBtn = QPushButton("删除主页")
		operatorLayout.addWidget(self.startBtn)
		operatorLayout.addWidget(self.stopBtn)
		operatorLayout.addWidget(self.addBtn)
		operatorLayout.addWidget(self.delBtn)

        # 按钮的信号槽连接
		self.startBtn.clicked.connect(self.startBtnFun)
		self.stopBtn.clicked.connect(self.stopBtnFun)
		self.addBtn.clicked.connect(self.addBtnFun)
		self.delBtn.clicked.connect(self.delBtnFun)
        
		self.resize(500,300);
		self.model=QStandardItemModel(0, 3);
		self.model.setHorizontalHeaderLabels(['主页URL', '主页名', '近半小时作品数'])
		#for i, html_path in enumerate(html_path_list):
		#	self.model.setItem(i, 0, QStandardItem("%s"%(html_path))) 

		
		self.tableView=QTableView()
		self.tableView.setModel(self.model)
		self.tableView.setItemDelegateForColumn(1, ReadOnlyDelegate())
		#self.tableView.setItemDelegateForColumn(2, ReadOnlyDelegate())
		#下面代码让表格100填满窗口
		self.tableView.horizontalHeader().setStretchLastSection(True)
		self.tableView.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
        
		self.timeLabel = QLabel("此次运行时间:")
		self.meLabel = QLabel("作者:chenyujing VX:15280006510")
		labelLayout = QHBoxLayout()
		labelLayout.addWidget(self.timeLabel)
		labelLayout.addWidget(self.meLabel)
        
		dlgLayout=QVBoxLayout()
		dlgLayout.addLayout(operatorLayout);
		dlgLayout.addWidget(self.tableView)
		dlgLayout.addLayout(labelLayout)
        
		self.setLayout(dlgLayout)
        
		# 创建运行时间的定时器
		self.Timer = QTimer()
    
	def reload(self, html_info_list = []):
		for i, html_info in enumerate(html_info_list):
			html_path, page_name, work_count = html_info
			self.model.setItem(i, 0, QStandardItem("%s"%(html_path))) 
			self.model.setItem(i, 1, QStandardItem("%s"%(page_name))) 
		update_xin_zeng_shu(self) 
        
	def addBtnFun(self):
		print('--- addBtnFun ---')
		value, ok = QInputDialog.getText(self, "添加主页", "请输入主页url:", QLineEdit.Normal, "")
		if ok and value:
			self.model.setItem(len(g_html_info_list), 0, QStandardItem("%s"%(value))) 
			g_html_info_list.append((value, "", 0))
		return
    
	def delBtnFun(self):
		print('--- delBtnFun ---')
		row = self.tableView.currentIndex().row()        #获取被选择的行数
		if -1 == row:
			QMessageBox.information(self, "提示", "请在表头选择整行后进行删除", QMessageBox.Yes)
			return
		print("移除第{}行".format(row))
		#indices = self.tableView.selectionModel().selectedRows() 
		#for index in sorted(indices):
		#	print("移除第{}行".format(index.row()))
		self.model.removeRow(row) 
 
	def startBtnFun(self):
		global g_html_info_list
		print('--- startBtnFun ---')
		count_row = self.model.rowCount()
		print("共有{}行".format(count_row))
		html_info_list = copy.deepcopy(g_html_info_list)
		g_html_info_list = []
		for i in range(count_row):
			url = self.model.data(self.model.index(i,0))
			print("获取的URL:{}".format(url))
			bFound = False
			for j, html_info in enumerate(html_info_list):
				html_path, page_name, work_count = html_info
				if url == html_path:
					bFound = True
					g_html_info_list.append((url, page_name, work_count))
			if bFound == False:
				g_html_info_list.append((url, "", 0))
		save_zuo_pin_shu_to_file(g_html_info_list, ZUO_PIN_INFO_CONFIG)
        
		g_Event.clear()
		#创建线程
		thread = threading.Thread(target = monitor_thread, args = (self, ))  
		thread.start()
        
		self.startBtn.setEnabled(False)
		self.addBtn.setEnabled(False)
		self.delBtn.setEnabled(False)
		#self.tableView.setEnabled(False)
        
		# 定时器每500ms工作一次
		self.Timer.start(500)
		# 建立定时器连接通道  注意这里调用TimeUpdate方法，不是方法返回的的结果，所以不能带括号，写成self.TimeUpdate()是不对的
		self.Timer.timeout.connect(self.TimeUpdate)
		self.datetimeStart = datetime.datetime.now()

		return
    
	def stopBtnFun(self):
		print('--- stopBtnFun ---')
		g_Event.set()
        
		self.startBtn.setEnabled(True)
		self.addBtn.setEnabled(True)
		self.delBtn.setEnabled(True)
		#self.tableView.setEnabled(True)
        
		self.Timer.stop()
		return
    
	def TimeUpdate(self):
		datetimeNow = datetime.datetime.now()
		intal_seconds = (datetimeNow - self.datetimeStart).seconds
		intal_hours = int(int(intal_seconds)/3600)
		intal_seconds = intal_seconds - 3600*intal_hours
		intal_minutes = int(int(intal_seconds)/60)
		intal_seconds = intal_seconds - 60*intal_minutes
		strDiplay = "监控正在进行:" + str(intal_hours) + "时" + str(intal_minutes) + "分" + str(intal_seconds) + "秒"
		self.timeLabel.setText(strDiplay)
 


# In[17]:


if __name__ == '__main__':
	app = QApplication(sys.argv)	
	table = Table()
	table.reload(g_html_info_list)
	table.show()
	sys.exit(app.exec_())

